SVM & Stock Price Prediction

0.1 0. Install Packages

pkgs <- c("jiebaR", "tidyverse", "tidytext", "stringr", "e1071", "tidyr", "Rtsne")
pkgs <- pkgs[!pkgs %in% installed.packages()[,"Package"]]
if(length(pkgs)) { install.packages(pkgs)}

library(tidyverse)
library(stringr)
options(stringsAsFactors = F)

1 Loading data

library(jiebaR)
segment_not <- c("鴻海" ,  "永豐金", "中信金", "台積電", "聯發科" ,"兆豐金", "台指期","郭台銘","張忠謀","鉅亨網")
cutter <- worker()
new_user_word(cutter,segment_not)
stopWords <- readRDS("data/stopWords.rds")

2 Stopwords

unnested.df <- stock_news %>%
    select(doc_id = newsId, text = content, status = status_p) %>%
    mutate(word = purrr::map(text, function(x)segment(x, cutter))) %>%
    unnest(word) %>%
    filter(!is.na(word)) %>% 
    filter(!word %in% stopWords$word) %>%
    filter(!str_detect(word, "[a-zA-Z0-9]+")) %>%
    filter(nchar(word) > 1)

• original dimension: > 610 news x 12,936 words

unnested.df %>%
    count(doc_id, word) %>%
    spread(word, n, fill = 0) %>% dim

[1]   610 12936

3 4. Chi-square feature selection

chi_df <- unnested.df %>%
    count(word, status) %>%
    filter(n > 3) %>%
    spread(status, n, fill = 0) %>%
    rename(A=`1`, C=`0`) %>%
    mutate(B=sum(A)-A,
           D=sum(C)-C,
           N=A+B+C+D, 
           chi2 = (A*D - B*C)^2 * N / ((A+C)*(A+B)*(B+D)*(C+D))) %>%
    filter(chi2 > 6.64)

4 5. Counting doc term frequency after feature selection

doc_term_count <- unnested.df %>%
    left_join(chi_df) %>%
    filter(!is.na(chi2)) %>%
    count(doc_id, word)

Joining, by = "word"

doc_term_count %>%
    spread(word, n, fill = 0) %>%
    dim

[1] 609 546

5 6. TF-IDF（term frequency & inverse document frequency）

# install.packages("tidytext")
# dtm <- cast_dtm(word_token, title, words, n)
# ??cast_dtm
comb.df <- doc_term_count %>%
    tidytext::bind_tf_idf(word, doc_id, n) %>%
    select(doc_id, word, tf_idf) %>%
    spread(word, tf_idf, fill=0) %>%
    left_join(select(stock_news, doc_id = newsId, status = status_p)) %>%
    select(doc_id, status, everything())

Joining, by = "doc_id"

6 7. T-SNE

library(Rtsne) # cannot be installed in MacOS mojave
tsne <- comb.df %>% select(-doc_id, -status) %>%
    Rtsne(perplexity = 35, dims = 2, check_duplicates = F)
# 取出降維後的特徵值df
feature_tsne <- comb.df %>%
    select(doc_id, status) %>%
    mutate(status = as.factor(status)) %>%
    bind_cols(as.data.frame(tsne$Y)) %>%
    mutate(id = row_number())

7 plotting tsne results

feature_tsne %>%
    ggplot() + aes(V1, V2, color = status) + 
    geom_point()

8 8. divide to tranining and testing set

set.seed(2017)

samples <- sample(1:nrow(feature_tsne), 
                  size = round(nrow(feature_tsne)*0.6))

trainset <- feature_tsne %>% select(-doc_id) %>% slice(samples)
testset <- feature_tsne[-samples,-1]

9 9. SVM

library(e1071)
model <- svm(status~ ., data = trainset, kernel="radial")
plot(model, trainset, V1~V2)

predicting  <- predict(model, testset %>% select(-status))
# creating confusion matrix
# https://en.wikipedia.org/wiki/Confusion_matrix
table(predicting, testset$status)

          
predicting   0   1
         0 126  61
         1  30  27

# accuracy
pre <- predicting == testset$status
percent1 <- length(pre[pre == T]) / length(pre)
percent1

[1] 0.6270492